Optimize the bakery lock structure for coherent memory

This patch optimizes the data structure used with the bakery lock
implementation for coherent memory to save memory and minimize memory
accesses. These optimizations were already part of the bakery lock
implementation for normal memory and this patch now implements
it for the coherent memory implementation as well. Also
included in the patch is a cleanup to use the do-while loop while
waiting for other contenders to finish choosing their tickets.

Change-Id: Iedb305473133dc8f12126726d8329b67888b70f1

diff --git a/include/lib/bakery_lock.h b/include/lib/bakery_lock.h
index 9736f850accdfd5c54bb8afa851a178dec524f32..eaefecd9e2334b0c56460c8de269a10194b2bea2 100644 (file)
--- a/include/lib/bakery_lock.h
+++ b/include/lib/bakery_lock.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2015, ARM Limited and Contributors. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -38,12 +38,34 @@
 #ifndef __ASSEMBLY__
 #include <stdint.h>
 
+/*****************************************************************************
+ * Internal helper macros used by the bakery lock implementation.
+ ****************************************************************************/
+/* Convert a ticket to priority */
+#define PRIORITY(t, pos)       (((t) << 8) | (pos))
+
+#define CHOOSING_TICKET                0x1
+#define CHOSEN_TICKET          0x0
+
+#define bakery_is_choosing(info)       (info & 0x1)
+#define bakery_ticket_number(info)     ((info >> 1) & 0x7FFF)
+#define make_bakery_data(choosing, number) \
+               (((choosing & 0x1) | (number << 1)) & 0xFFFF)
+
+/*****************************************************************************
+ * External bakery lock interface.
+ ****************************************************************************/
 #if USE_COHERENT_MEM
 
 typedef struct bakery_lock {
        int owner;
-       volatile char entering[BAKERY_LOCK_MAX_CPUS];
-       volatile unsigned number[BAKERY_LOCK_MAX_CPUS];
+       /*
+        * The lock_data is a bit-field of 2 members:
+        * Bit[0]       : choosing. This field is set when the CPU is
+        *                choosing its bakery number.
+        * Bits[1 - 15] : number. This is the bakery number allocated.
+        */
+       volatile uint16_t lock_data[BAKERY_LOCK_MAX_CPUS];
 } bakery_lock_t;
 
 #define NO_OWNER (-1)

diff --git a/lib/locks/bakery/bakery_lock_coherent.c b/lib/locks/bakery/bakery_lock_coherent.c
index 5d538ce2c6afc301ed9d0ec6f92c2121f0ced92d..02d31d6463999a0fd6be6e63d7d22d07200e5828 100644 (file)
--- a/lib/locks/bakery/bakery_lock_coherent.c
+++ b/lib/locks/bakery/bakery_lock_coherent.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2015, ARM Limited and Contributors. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -63,10 +63,6 @@
        assert(entry < BAKERY_LOCK_MAX_CPUS);           \
 } while (0)
 
-/* Convert a ticket to priority */
-#define PRIORITY(t, pos)       (((t) << 8) | (pos))
-
-
 /* Initialize Bakery Lock to reset ownership and all ticket values */
 void bakery_lock_init(bakery_lock_t *bakery)
 {
@@ -95,9 +91,9 @@ static unsigned int bakery_get_ticket(bakery_lock_t *bakery, unsigned int me)
         * value, not the ticket value alone.
         */
        my_ticket = 0;
-       bakery->entering[me] = 1;
+       bakery->lock_data[me] = make_bakery_data(CHOOSING_TICKET, my_ticket);
        for (they = 0; they < BAKERY_LOCK_MAX_CPUS; they++) {
-               their_ticket = bakery->number[they];
+               their_ticket = bakery_ticket_number(bakery->lock_data[they]);
                if (their_ticket > my_ticket)
                        my_ticket = their_ticket;
        }
@@ -107,8 +103,7 @@ static unsigned int bakery_get_ticket(bakery_lock_t *bakery, unsigned int me)
         * finish calculating our ticket value that we're done
         */
        ++my_ticket;
-       bakery->number[me] = my_ticket;
-       bakery->entering[me] = 0;
+       bakery->lock_data[me] = make_bakery_data(CHOSEN_TICKET, my_ticket);
 
        return my_ticket;
 }
@@ -129,6 +124,7 @@ void bakery_lock_get(bakery_lock_t *bakery)
 {
        unsigned int they, me;
        unsigned int my_ticket, my_prio, their_ticket;
+       unsigned int their_bakery_data;
 
        me = platform_get_core_pos(read_mpidr_el1());
 
@@ -150,14 +146,15 @@ void bakery_lock_get(bakery_lock_t *bakery)
                        continue;
 
                /* Wait for the contender to get their ticket */
-               while (bakery->entering[they])
-                       ;
+               do {
+                       their_bakery_data = bakery->lock_data[they];
+               } while (bakery_is_choosing(their_bakery_data));
 
                /*
                 * If the other party is a contender, they'll have non-zero
                 * (valid) ticket value. If they do, compare priorities
                 */
-               their_ticket = bakery->number[they];
+               their_ticket = bakery_ticket_number(their_bakery_data);
                if (their_ticket && (PRIORITY(their_ticket, they) < my_prio)) {
                        /*
                         * They have higher priority (lower value). Wait for
@@ -167,7 +164,8 @@ void bakery_lock_get(bakery_lock_t *bakery)
                         */
                        do {
                                wfe();
-                       } while (their_ticket == bakery->number[they]);
+                       } while (their_ticket ==
+                               bakery_ticket_number(bakery->lock_data[they]));
                }
        }
 
@@ -189,7 +187,7 @@ void bakery_lock_release(bakery_lock_t *bakery)
         * waiting contenders
         */
        bakery->owner = NO_OWNER;
-       bakery->number[me] = 0;
+       bakery->lock_data[me] = 0;
        dsb();
        sev();
 }

diff --git a/lib/locks/bakery/bakery_lock_normal.c b/lib/locks/bakery/bakery_lock_normal.c
index a325fd4feb2716ceff07c2ae9f2a39c948218b11..4503ae09740c2d93bdfa0561a8ea4175a5bb4ece 100644 (file)
--- a/lib/locks/bakery/bakery_lock_normal.c
+++ b/lib/locks/bakery/bakery_lock_normal.c
@@ -56,17 +56,6 @@
  * accesses regardless of status of address translation.
  */
 
-/* Convert a ticket to priority */
-#define PRIORITY(t, pos)       (((t) << 8) | (pos))
-
-#define CHOOSING_TICKET                0x1
-#define CHOOSING_DONE          0x0
-
-#define bakery_is_choosing(info)       (info & 0x1)
-#define bakery_ticket_number(info)     ((info >> 1) & 0x7FFF)
-#define make_bakery_data(choosing, number) \
-               (((choosing & 0x1) | (number << 1)) & 0xFFFF)
-
 /* This macro assumes that the bakery_info array is located at the offset specified */
 #define get_my_bakery_info(offset, id)         \
        (((bakery_info_t *) (((uint8_t *)_cpu_data()) + offset)) + id)
@@ -138,7 +127,7 @@ static unsigned int bakery_get_ticket(int id, unsigned int offset,
         * finish calculating our ticket value that we're done
         */
        ++my_ticket;
-       my_bakery_info->lock_data = make_bakery_data(CHOOSING_DONE, my_ticket);
+       my_bakery_info->lock_data = make_bakery_data(CHOSEN_TICKET, my_ticket);
 
        write_cache_op(my_bakery_info, is_cached);
 
@@ -150,7 +139,7 @@ void bakery_lock_get(unsigned int id, unsigned int offset)
        unsigned int they, me, is_cached;
        unsigned int my_ticket, my_prio, their_ticket;
        bakery_info_t *their_bakery_info;
-       uint16_t their_bakery_data;
+       unsigned int their_bakery_data;
 
        me = platform_get_core_pos(read_mpidr_el1());
 
@@ -174,15 +163,12 @@ void bakery_lock_get(unsigned int id, unsigned int offset)
                 */
                their_bakery_info = get_bakery_info_by_index(offset, id, they);
                assert(their_bakery_info);
-               read_cache_op(their_bakery_info, is_cached);
-
-               their_bakery_data = their_bakery_info->lock_data;
 
                /* Wait for the contender to get their ticket */
-               while (bakery_is_choosing(their_bakery_data)) {
+               do {
                        read_cache_op(their_bakery_info, is_cached);
                        their_bakery_data = their_bakery_info->lock_data;
-               }
+               } while (bakery_is_choosing(their_bakery_data));
 
                /*
                 * If the other party is a contender, they'll have non-zero